// TODO 实现多个字符的SYMBOL识别
#include <vector>
#include <string>
#include <cstdlib>
#include <cctype>
#include <climits>
#define NUMBER 	1
#define STRING 	2
#define NAME 	3
#define SYMBOL	4
#define NEWLINE	5
using namespace std;

vector<string> split(string code) {
	vector<string> lines;
	string now;
	for (int i = 0; i < code.size(); i++) {
		if (code[i] == '\n') {
			lines.push_back(now);
			now = string();
		} else now += code[i];
	}
	return lines;
}

class Tokenizer {
	private:
		const char table[9][2] = {
			{'a', '\a'},
			{'b', '\b'},
			{'f', '\f'},
			{'n', '\n'},
			{'r', '\r'},
			{'t', '\t'},
			{'v', '\v'},
			{'0', '\0'}
		};
		struct Token {
			int type;
			string value;
			int line, col;
		};
		string delComments(string code) {
			int cur = 0, size = code.size();
			string newcode;
			char q = 0;
			while(cur < size) {
				if (!q && code[cur] == '#') {
					while(code[cur] != '\n') cur++;
					newcode += '\n';
				} else if (q && code[cur] == '\\') {
					newcode += code[cur++];
					newcode += code[cur];
				} else if (q && q == code[cur]) {
					q = 0;
					newcode += code[cur];
				} else if (!q && code[cur] == '\'' || code[cur] == '"') {
					q = code[cur];
					newcode += code[cur];
				} else newcode += code[cur];
				cur++;
			}
			return newcode;
		}
		string filename, code, value;
		int line, col, cur, type, tline, tcol, pointcnt, ecnt, xcnt, size, maxline;
		bool is16;
		bool flag;
		char quote;
		vector<string> lines;
#define clear() (pointcnt = ecnt = is16 = xcnt = 0)
		void nxt() {
			if (cur + 1 > size) error("unexpected EOF", 1);
			if (code[cur++] == '\n') line++, col = 1;
			else col++;
		}
		void error(const char s[], bool isexit = 0) {
			if (line > maxline) line = maxline, col = INT_MAX;
			fprintf(stderr, "%s:%d:%d: syntax error: %s\n\t%s\n\t", filename.c_str(), line, col == INT_MAX ? 1 : col, s, lines[line - 1].c_str());
			for (int i = 1; i < col && i < lines[line - 1].size(); i++) fputc(' ', stderr);
			fputs("^\n", stderr);
			if (isexit) exit(1);
			flag = type = 0;
			while(code[cur] == '_' || (!ispunct(code[cur]) && !isblank(code[cur]))) nxt();
			clear();
		}
		void add() {
			list.push_back((Token){type, value, tline, tcol});
			type = 0;
			value = string();
			clear();
		}
		void add(int a, string b, int c, int d) {
			list.push_back((Token){a, b, c, d});
			type = 0;
			value = string();
			clear();
		}
	public:
		vector<Token> list;
		Tokenizer(string _code, vector<string> _lines, string _filename): code(delComments(_code)), lines(_lines), filename(_filename) {
			line = col = flag = 1;
			cur = type = size = 0;
			maxline = lines.size();
			clear();
			size = code.size();
			while (cur < size) {
				char c = code[cur];
				if (isalpha(c)) {
					if (type == NUMBER) {
						if (c == 'e' || c == 'E') {
							ecnt++;
							if (ecnt > 1) error("extra exponent");
							value += c;
						} else if (c == 'X' || c == 'x') {
							is16 = 1;
							xcnt++;
							if (xcnt > 1) error("extra 'x'");
							value += c;
						} else if (is16) {
							if (c >= 'a' && c <= 'f' || c >= 'A' && c <= 'F') value += c;
							else error("hexadecimal can only contain letters a-f");
						} else error("unexpected letter after number");
					} else if (type == STRING || type == NAME)
						value += c;
					else if (!type) {
						tline = line;
						tcol = col;
						type = NAME;
						value += c;
					}
				} else if (isdigit(c)) {
					if (type == NUMBER || type == STRING || type == NAME)
						value += c;
					else if (!type) {
						tline = line;
						tcol = col;
						type = NUMBER;
						value += c;
					}
				} else if (ispunct(c)) {
					if (type == STRING && c != quote) {
						if (c == '\\') {
							nxt();
							char after = code[cur];
							for (int i = 0; i < 9; i++) 
								if (table[i][0] == after) {
									value += table[i][1];
									goto done;
								}
							value += after;
							done:;
						} else value += c;
					} else if (type == NUMBER) {
						if (c == '.') {
							if (ecnt > 1) error("exponent cannot be a floating point number");
							pointcnt++;
							if (pointcnt > 1) error("extra decimal point");
							value += c;
						} else {
							if ((c == '-' || c == '+') && (cur > 0 && code[cur - 1] == 'e' || code[cur - 1] == 'E')) value += c;
							else {
								if (code[cur - 1] == '-' || code[cur - 1] == '+') error("missing number");
								add();
								const char a[2] = {c};
								add(SYMBOL, a, line, col);
							}
						}
					} else if (c == '"' || c == '\'') {
						quote = c;
						if (!type || type != STRING) {
							if (type) add();
							type = STRING;
							tline = line;
							tcol = col;
						} else add();
					} else {
						if (type && c != '_') add();
						if (c == '.') {
							if (cur + 1 < size) {
								if (isdigit(code[cur + 1])) {
									type = NUMBER;
									value += c;
									tline = line;
									tcol = col;
								} else {
									const char a[2] = {c};
									add(SYMBOL, a, line, col);
								}
							} else error("extra point");
						} else if (c == '_') value += c;
						else {
							const char a[2] = {c};
							add(SYMBOL, a, line, col);
						}
					}
				} else if (isblank(c)) {
					if (type == STRING) value += c;	
					else if (type) {
						if (type == NUMBER && !isdigit(code[cur - 1])) error("missing number");
						add();
					}
				} else if (c == '\n') {
					if (type == STRING) value += c;
					else {
						if (type) add();
						add(NEWLINE, string(), line, col);
					}
				} else if (type == STRING) value += c;
				nxt();
			}
			if (type == STRING) error("missing quotes", 1);
			if (!flag) exit(1);
		}
};
